/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.handler;

import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexCommit;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.FileUtils;
import org.apache.solr.core.SolrCore;
import static org.apache.solr.handler.ReplicationHandler.*;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.Adler32;
import java.util.zip.Checksum;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;

/**
 * <p/> Provides functionality of downloading changed index files as well as config files and a timer for scheduling fetches from the
 * master. </p>
 *
 * @version $Id: SnapPuller.java 1065312 2011-01-30 16:08:25Z rmuir $
 * @since solr 1.4
 */

public class SnapPuller{}
//
//public class SnapPuller {
//  private static final Logger LOG = LoggerFactory.getLogger(SnapPuller.class.getName());
//
//  private final String masterUrl;
//
//  private final ReplicationHandler replicationHandler;
//
//  private final Integer pollInterval;
//
//  private String pollIntervalStr;
//
//  private ScheduledExecutorService executorService;
//
//  private volatile long executorStartTime;
//
//  private volatile long replicationStartTime;
//
//  private final SolrCore solrCore;
//
//  private volatile List<Map<String, Object>> filesToDownload;
//
//  private volatile List<Map<String, Object>> confFilesToDownload;
//
//  private volatile List<Map<String, Object>> filesDownloaded;
//
//  private volatile List<Map<String, Object>> confFilesDownloaded;
//
//  private volatile Map<String, Object> currentFile;
//
//  private volatile FileFetcher fileFetcher;
//
//  private volatile ExecutorService fsyncService;
//
//  private volatile boolean stop = false;
//
//  private boolean useInternal = false;
//
//  private boolean useExternal = false;
//
//  /**
//   * Disable the timer task for polling
//   */
//  private AtomicBoolean pollDisabled = new AtomicBoolean(false);
//
//  // HttpClient shared by all cores (used if timeout is not specified for a core)
//  private static HttpClient client;
//  // HttpClient for this instance if connectionTimeout or readTimeout has been specified
//  private final HttpClient myHttpClient;
//
//  private static synchronized HttpClient createHttpClient(String connTimeout, String readTimeout) {
//    if (connTimeout == null && readTimeout == null && client != null)  return client;
//    MultiThreadedHttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
//    // Keeping a very high number so that if you have a large number of cores
//    // no requests are kept waiting for an idle connection.
//    mgr.getParams().setDefaultMaxConnectionsPerHost(10000);
//    mgr.getParams().setMaxTotalConnections(10000);
//    mgr.getParams().setSoTimeout(readTimeout == null ? 20000 : Integer.parseInt(readTimeout)); //20 secs
//    mgr.getParams().setConnectionTimeout(connTimeout == null ? 5000 : Integer.parseInt(connTimeout)); //5 secs
//    HttpClient httpClient = new HttpClient(mgr);
//    if (client == null && connTimeout == null && readTimeout == null) client = httpClient;
//    return httpClient;
//  }
//
//  public SnapPuller(NamedList initArgs, ReplicationHandler handler, SolrCore sc) {
//    solrCore = sc;
//    masterUrl = (String) initArgs.get(MASTER_URL);
//    if (masterUrl == null)
//      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//              "'masterUrl' is required for a slave");
//    this.replicationHandler = handler;
//    pollIntervalStr = (String) initArgs.get(POLL_INTERVAL);
//    pollInterval = readInterval(pollIntervalStr);
//    String compress = (String) initArgs.get(COMPRESSION);
//    useInternal = INTERNAL.equals(compress);
//    useExternal = EXTERNAL.equals(compress);
//    String connTimeout = (String) initArgs.get(HTTP_CONN_TIMEOUT);
//    String readTimeout = (String) initArgs.get(HTTP_READ_TIMEOUT);
//    String httpBasicAuthUser = (String) initArgs.get(HTTP_BASIC_AUTH_USER);
//    String httpBasicAuthPassword = (String) initArgs.get(HTTP_BASIC_AUTH_PASSWORD);
//    myHttpClient = createHttpClient(connTimeout, readTimeout);
//    if (httpBasicAuthUser != null && httpBasicAuthPassword != null) {
//      myHttpClient.getState().setCredentials(AuthScope.ANY,
//              new UsernamePasswordCredentials(httpBasicAuthUser, httpBasicAuthPassword));
//    }
//    if (pollInterval != null && pollInterval > 0) {
//      startExecutorService();
//    } else {
//      LOG.info(" No value set for 'pollInterval'. Timer Task not started.");
//    }
//  }
//
//  private void startExecutorService() {
//    Runnable task = new Runnable() {
//      public void run() {
//        if (pollDisabled.get()) {
//          LOG.info("Poll disabled");
//          return;
//        }
//        try {
//          executorStartTime = System.currentTimeMillis();
//          replicationHandler.doFetch(null);
//        } catch (Exception e) {
//          LOG.error("Exception in fetching index", e);
//        }
//      }
//    };
//    executorService = Executors.newSingleThreadScheduledExecutor();
//    long initialDelay = pollInterval - (System.currentTimeMillis() % pollInterval);
//    executorService.scheduleAtFixedRate(task, initialDelay, pollInterval, TimeUnit.MILLISECONDS);
//    LOG.info("Poll Scheduled at an interval of " + pollInterval + "ms");
//  }
//
//  /**
//   * Gets the latest commit version and generation from the master
//   */
//  @SuppressWarnings("unchecked")
//  NamedList getLatestVersion() throws IOException {
//    PostMethod post = new PostMethod(masterUrl);
//    post.addParameter(COMMAND, CMD_INDEX_VERSION);
//    post.addParameter("wt", "javabin");
//    return getNamedListResponse(post);
//  }
//
//  NamedList getCommandResponse(NamedList<String> commands) throws IOException {
//    PostMethod post = new PostMethod(masterUrl);
//    for (Map.Entry<String, String> c : commands) {
//      post.addParameter(c.getKey(),c.getValue());
//    }
//    post.addParameter("wt", "javabin");
//    return getNamedListResponse(post);
//  }
//
//  private NamedList getNamedListResponse(PostMethod method) throws IOException {
//    try {
//      int status = myHttpClient.executeMethod(method);
//      if (status != HttpStatus.SC_OK) {
//        throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
//                "Request failed for the url " + method);
//      }
//      return (NamedList) new JavaBinCodec().unmarshal(method.getResponseBodyAsStream());
//    } finally {
//      try {
//        method.releaseConnection();
//      } catch (Exception e) {
//      }
//    }
//  }
//
//  /**
//   * Fetches the list of files in a given index commit point
//   */
//  void fetchFileList(long version) throws IOException {
//    PostMethod post = new PostMethod(masterUrl);
//    post.addParameter(COMMAND, CMD_GET_FILE_LIST);
//    post.addParameter(CMD_INDEX_VERSION, String.valueOf(version));
//    post.addParameter("wt", "javabin");
//    NamedList nl = getNamedListResponse(post);
//    List<Map<String, Object>> f = (List<Map<String, Object>>) nl.get(CMD_GET_FILE_LIST);
//    if (f != null)
//      filesToDownload = Collections.synchronizedList(f);
//    else {
//      filesToDownload = Collections.emptyList();
//      LOG.error("No files to download for indexversion: "+ version);
//    }
//
//    f = (List<Map<String, Object>>) nl.get(CONF_FILES);
//    if (f != null)
//      confFilesToDownload = Collections.synchronizedList(f);
//  }
//
//  /**
//   * This command downloads all the necessary files from master to install a index commit point. Only changed files are
//   * downloaded. It also downloads the conf files (if they are modified).
//   *
//   * @param core the SolrCore
//   * @return true on success, false if slave is already in sync
//   * @throws IOException if an exception occurs
//   */
//  @SuppressWarnings("unchecked")
//  boolean successfulInstall = false;
//
//  boolean fetchLatestIndex(SolrCore core) throws IOException {
//    replicationStartTime = System.currentTimeMillis();
//    try {
//      //get the current 'replicateable' index version in the master
//      NamedList response = null;
//      try {
//        response = getLatestVersion();
//      } catch (Exception e) {
//        LOG.error("Master at: " + masterUrl + " is not available. Index fetch failed. Exception: " + e.getMessage());
//        return false;
//      }
//      long latestVersion = (Long) response.get(CMD_INDEX_VERSION);
//      long latestGeneration = (Long) response.get(GENERATION);
//      if (latestVersion == 0L) {
//        //there is nothing to be replicated
//        return false;
//      }
//      IndexCommit commit;
//      RefCounted<SolrIndexSearcher> searcherRefCounted = null;
//      try {
//        searcherRefCounted = core.getNewestSearcher(false);
//        commit = searcherRefCounted.get().getReader().getIndexCommit();
//      } finally {
//        if (searcherRefCounted != null)
//          searcherRefCounted.decref();
//      }
//      if (commit.getVersion() == latestVersion && commit.getGeneration() == latestGeneration) {
//        //master and slave are alsready in sync just return
//        LOG.info("Slave in sync with master.");
//        return false;
//      }
//      LOG.info("Master's version: " + latestVersion + ", generation: " + latestGeneration);
//      LOG.info("Slave's version: " + commit.getVersion() + ", generation: " + commit.getGeneration());
//      LOG.info("Starting replication process");
//      // get the list of files first
//      fetchFileList(latestVersion);
//      // this can happen if the commit point is deleted before we fetch the file list.
//      if(filesToDownload.isEmpty()) return false;
//      LOG.info("Number of files in latest index in master: " + filesToDownload.size());
//
//      // Create the sync service
//      fsyncService = Executors.newSingleThreadExecutor();
//      // use a synchronized list because the list is read by other threads (to show details)
//      filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
//      // if the generateion of master is older than that of the slave , it means they are not compatible to be copied
//      // then a new index direcory to be created and all the files need to be copied
//      boolean isFullCopyNeeded = commit.getGeneration() >= latestGeneration;
//      File tmpIndexDir = createTempindexDir(core);
//      if (isIndexStale())
//        isFullCopyNeeded = true;
//      successfulInstall = false;
//      boolean deleteTmpIdxDir = true;
//      File indexDir = null ;
//      try {
//        indexDir = new File(core.getIndexDir());
//        downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestVersion);
//        LOG.info("Total time taken for download : " + ((System.currentTimeMillis() - replicationStartTime) / 1000) + " secs");
//        Collection<Map<String, Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
//        if (!modifiedConfFiles.isEmpty()) {
//          downloadConfFiles(confFilesToDownload, latestVersion);
//          if (isFullCopyNeeded) {
//            successfulInstall = modifyIndexProps(tmpIndexDir.getName());
//            deleteTmpIdxDir =  false;
//          } else {
//            successfulInstall = copyIndexFiles(tmpIndexDir, indexDir);
//          }
//          if (successfulInstall) {
//            LOG.info("Configuration files are modified, core will be reloaded");
//            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);//write to a file time of replication and conf files.
//            reloadCore();
//          }
//        } else {
//          terminateAndWaitFsyncService();
//          if (isFullCopyNeeded) {
//            successfulInstall = modifyIndexProps(tmpIndexDir.getName());
//            deleteTmpIdxDir =  false;
//          } else {
//            successfulInstall = copyIndexFiles(tmpIndexDir, indexDir);
//          }
//          if (successfulInstall) {
//            logReplicationTimeAndConfFiles(modifiedConfFiles, successfulInstall);
//            doCommit();
//          }
//        }
//        replicationStartTime = 0;
//        return successfulInstall;
//      } catch (ReplicationHandlerException e) {
//        LOG.error("User aborted Replication");
//      } catch (SolrException e) {
//        throw e;
//      } catch (Exception e) {
//        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Index fetch failed : ", e);
//      } finally {
//        if (deleteTmpIdxDir) delTree(tmpIndexDir);
//        else delTree(indexDir);
//      }
//      return successfulInstall;
//    } finally {
//      if (!successfulInstall) {
//        logReplicationTimeAndConfFiles(null, successfulInstall);
//      }
//      filesToDownload = filesDownloaded = confFilesDownloaded = confFilesToDownload = null;
//      replicationStartTime = 0;
//      fileFetcher = null;
//      if (fsyncService != null && !fsyncService.isShutdown()) fsyncService.shutdownNow();
//      fsyncService = null;
//      stop = false;
//      fsyncException = null;
//    }
//  }
//
//  private volatile Exception fsyncException;
//
//  /**
//   * terminate the fsync service and wait for all the tasks to complete. If it is already terminated
//   *
//   * @throws Exception
//   */
//  private void terminateAndWaitFsyncService() throws Exception {
//    if (fsyncService.isTerminated()) return;
//    fsyncService.shutdown();
//     // give a long wait say 1 hr
//    fsyncService.awaitTermination(3600, TimeUnit.SECONDS);
//    // if any fsync failed, throw that exception back
//    Exception fsyncExceptionCopy = fsyncException;
//    if (fsyncExceptionCopy != null) throw fsyncExceptionCopy;
//  }
//
//  /**
//   * Helper method to record the last replication's details so that we can show them on the statistics page across
//   * restarts.
//   */
//  private void logReplicationTimeAndConfFiles(Collection<Map<String, Object>> modifiedConfFiles, boolean successfulInstall) {
//    FileOutputStream outFile = null;
//    List<String> confFiles = new ArrayList<String>();
//    if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty())
//      for (Map<String, Object> map1 : modifiedConfFiles)
//        confFiles.add((String) map1.get(NAME));
//
//    Properties props = replicationHandler.loadReplicationProperties();
//    long replicationTime = System.currentTimeMillis();
//    long replicationTimeTaken = (replicationTime - getReplicationStartTime()) / 1000;
//    try {
//      int indexCount = 1, confFilesCount = 1;
//      if (props.containsKey(TIMES_INDEX_REPLICATED)) {
//        indexCount = Integer.valueOf(props.getProperty(TIMES_INDEX_REPLICATED)) + 1;
//      }
//      StringBuffer sb = readToStringBuffer(replicationTime, props.getProperty(INDEX_REPLICATED_AT_LIST));
//      props.setProperty(INDEX_REPLICATED_AT_LIST, sb.toString());
//      props.setProperty(INDEX_REPLICATED_AT, String.valueOf(replicationTime));
//      props.setProperty(PREVIOUS_CYCLE_TIME_TAKEN, String.valueOf(replicationTimeTaken));
//      props.setProperty(TIMES_INDEX_REPLICATED, String.valueOf(indexCount));
//      if (modifiedConfFiles != null && !modifiedConfFiles.isEmpty()) {
//        props.setProperty(CONF_FILES_REPLICATED, confFiles.toString());
//        props.setProperty(CONF_FILES_REPLICATED_AT, String.valueOf(replicationTime));
//        if (props.containsKey(TIMES_CONFIG_REPLICATED)) {
//          confFilesCount = Integer.valueOf(props.getProperty(TIMES_CONFIG_REPLICATED)) + 1;
//        }
//        props.setProperty(TIMES_CONFIG_REPLICATED, String.valueOf(confFilesCount));
//      }
//
//      props.setProperty(LAST_CYCLE_BYTES_DOWNLOADED, String.valueOf(getTotalBytesDownloaded(this)));
//      if (!successfulInstall) {
//        int numFailures = 1;
//        if (props.containsKey(TIMES_FAILED)) {
//          numFailures = Integer.valueOf(props.getProperty(TIMES_FAILED)) + 1;
//        }
//        props.setProperty(TIMES_FAILED, String.valueOf(numFailures));
//        props.setProperty(REPLICATION_FAILED_AT, String.valueOf(replicationTime));
//        sb = readToStringBuffer(replicationTime, props.getProperty(REPLICATION_FAILED_AT_LIST));
//        props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
//      }
//      File f = new File(solrCore.getDataDir(), REPLICATION_PROPERTIES);
//      outFile = new FileOutputStream(f);
//      props.store(outFile, "Replication details");
//      outFile.close();
//    } catch (Exception e) {
//      LOG.warn("Exception while updating statistics", e);
//    }
//    finally {
//      IOUtils.closeQuietly(outFile);
//    }
//  }
//
//  static long getTotalBytesDownloaded(SnapPuller snappuller) {
//    long bytesDownloaded = 0;
//    //get size from list of files to download
//    for (Map<String, Object> file : snappuller.getFilesDownloaded()) {
//      bytesDownloaded += (Long) file.get(SIZE);
//    }
//
//    //get size from list of conf files to download
//    for (Map<String, Object> file : snappuller.getConfFilesDownloaded()) {
//      bytesDownloaded += (Long) file.get(SIZE);
//    }
//
//    //get size from current file being downloaded
//    Map<String, Object> currentFile = snappuller.getCurrentFile();
//    if (currentFile != null) {
//      if (currentFile.containsKey("bytesDownloaded")) {
//        bytesDownloaded += (Long) currentFile.get("bytesDownloaded");
//      }
//    }
//    return bytesDownloaded;
//  }
//
//  private StringBuffer readToStringBuffer(long replicationTime, String str) {
//    StringBuffer sb = new StringBuffer();
//    List<String> l = new ArrayList<String>();
//    if (str != null && str.length() != 0) {
//      String[] ss = str.split(",");
//      for (int i = 0; i < ss.length; i++) {
//        l.add(ss[i]);
//      }
//    }
//    sb.append(replicationTime);
//    if (!l.isEmpty()) {
//      for (int i = 0; i < l.size() || i < 9; i++) {
//        if (i == l.size() || i == 9) break;
//        String s = l.get(i);
//        sb.append(",").append(s);
//      }
//    }
//    return sb;
//  }
//
//  private void doCommit() throws IOException {
//    CommitUpdateCommand cmd = new CommitUpdateCommand(false);
//    cmd.waitFlush = true;
//    cmd.waitSearcher = true;
//    solrCore.getUpdateHandler().commit(cmd);
//    if (solrCore.getUpdateHandler() instanceof DirectUpdateHandler2) {
//      LOG.info("Force open index writer to make sure older index files get deleted");
//      DirectUpdateHandler2 handler = (DirectUpdateHandler2) solrCore.getUpdateHandler();
//      handler.forceOpenWriter();
//      replicationHandler.refreshCommitpoint();
//    } else  {
//      LOG.warn("The update handler is not an instance or sub-class of DirectUpdateHandler2. " +
//              "ReplicationHandler may not be able to cleanup un-used index files.");
//    }
//  }
//
//
//  /**
//   * All the files are copied to a temp dir first
//   */
//  private File createTempindexDir(SolrCore core) {
//    String tmpIdxDirName = "index." + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.US).format(new Date());
//    File tmpIdxDir = new File(core.getDataDir(), tmpIdxDirName);
//    tmpIdxDir.mkdirs();
//    return tmpIdxDir;
//  }
//
//  private void reloadCore() {
//    new Thread() {
//      @Override
//      public void run() {
//        try {
//          solrCore.getCoreDescriptor().getCoreContainer().reload(solrCore.getName());
//        } catch (Exception e) {
//          LOG.error("Could not restart core ", e);
//        }
//      }
//    }.start();
//  }
//
//  private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestVersion) throws Exception {
//    LOG.info("Starting download of configuration files from master: " + confFilesToDownload);
//    confFilesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
//    File tmpconfDir = new File(solrCore.getResourceLoader().getConfigDir(), "conf." + getDateAsStr(new Date()));
//    try {
//      boolean status = tmpconfDir.mkdirs();
//      if (!status) {
//        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                "Failed to create temporary config folder: " + tmpconfDir.getName());
//      }
//      for (Map<String, Object> file : confFilesToDownload) {
//        String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS));
//        fileFetcher = new FileFetcher(tmpconfDir, file, saveAs, true, latestVersion);
//        currentFile = file;
//        fileFetcher.fetchFile();
//        confFilesDownloaded.add(new HashMap<String, Object>(file));
//      }
//      // this is called before copying the files to the original conf dir
//      // so that if there is an exception avoid corrupting the original files.
//      terminateAndWaitFsyncService();
//      copyTmpConfFiles2Conf(tmpconfDir);
//    } finally {
//      delTree(tmpconfDir);
//    }
//  }
//
//  /**
//   * Download the index files. If a new index is needed, download all the files.
//   *
//   * @param downloadCompleteIndex is it a fresh index copy
//   * @param tmpIdxDir               the directory to which files need to be downloadeed to
//   * @param latestVersion         the version number
//   */
//  private void downloadIndexFiles(boolean downloadCompleteIndex, File tmpIdxDir, long latestVersion) throws Exception {
//    for (Map<String, Object> file : filesToDownload) {
//      File localIndexFile = new File(solrCore.getIndexDir(), (String) file.get(NAME));
//      if (!localIndexFile.exists() || downloadCompleteIndex) {
//        fileFetcher = new FileFetcher(tmpIdxDir, file, (String) file.get(NAME), false, latestVersion);
//        currentFile = file;
//        fileFetcher.fetchFile();
//        filesDownloaded.add(new HashMap<String, Object>(file));
//      } else {
//        LOG.info("Skipping download for " + localIndexFile);
//      }
//    }
//  }
//
//  /**
//   * All the files which are common between master and slave must have same timestamp and size else we assume they are
//   * not compatible (stale).
//   *
//   * @return true if the index stale and we need to download a fresh copy, false otherwise.
//   */
//  private boolean isIndexStale() {
//    for (Map<String, Object> file : filesToDownload) {
//      File localIndexFile = new File(solrCore.getIndexDir(), (String) file
//              .get(NAME));
//      if (localIndexFile.exists()
//              && localIndexFile.length() != (Long) file.get(SIZE)) {
//        // file exists and size is different, therefore we must assume
//        // corrupted index
//        return true;
//      }
//    }
//    return false;
//  }
//
//  /**
//   * Copy a file by the File#renameTo() method. If it fails, it is considered a failure
//   * <p/>
//   */
//  private boolean copyAFile(File tmpIdxDir, File indexDir, String fname, List<String> copiedfiles) {
//    File indexFileInTmpDir = new File(tmpIdxDir, fname);
//    File indexFileInIndex = new File(indexDir, fname);
//    boolean success = indexFileInTmpDir.renameTo(indexFileInIndex);
//    if(!success){
//      try {
//        LOG.error("Unable to move index file from: " + indexFileInTmpDir
//              + " to: " + indexFileInIndex + "Trying to do a copy");
//        FileUtils.copyFile(indexFileInTmpDir,indexFileInIndex);
//        success = true;
//      } catch (IOException e) {
//        LOG.error("Unable to copy index file from: " + indexFileInTmpDir
//              + " to: " + indexFileInIndex , e);
//      }
//    }
//    if (!success) {
//      for (String f : copiedfiles) {
//        File indexFile = new File(indexDir, f);
//        if (indexFile.exists())
//          indexFile.delete();
//      }
//      delTree(tmpIdxDir);
//      return false;
//    }
//    return true;
//  }
//
//  /**
//   * Copy all index files from the temp index dir to the actual index. The segments_N file is copied last.
//   */
//  private boolean copyIndexFiles(File tmpIdxDir, File indexDir) throws IOException {
//    String segmentsFile = null;
//    List<String> copiedfiles = new ArrayList<String>();
//    for (Map<String, Object> f : filesDownloaded) {
//      String fname = (String) f.get(NAME);
//      // the segments file must be copied last
//      // or else if there is a failure in between the
//      // index will be corrupted
//      if (fname.startsWith("segments_")) {
//        //The segments file must be copied in the end
//        //Otherwise , if the copy fails index ends up corrupted
//        segmentsFile = fname;
//        continue;
//      }
//      if (!copyAFile(tmpIdxDir, indexDir, fname, copiedfiles)) return false;
//      copiedfiles.add(fname);
//    }
//    //copy the segments file last
//    if (segmentsFile != null) {
//      if (!copyAFile(tmpIdxDir, indexDir, segmentsFile, copiedfiles)) return false;
//    }
//    return true;
//  }
//
//  /**
//   * The conf files are copied to the tmp dir to the conf dir. A backup of the old file is maintained
//   */
//  private void copyTmpConfFiles2Conf(File tmpconfDir) throws IOException {
//    File confDir = new File(solrCore.getResourceLoader().getConfigDir());
//    for (File file : tmpconfDir.listFiles()) {
//      File oldFile = new File(confDir, file.getName());
//      if (oldFile.exists()) {
//        File backupFile = new File(confDir, oldFile.getName() + "." + getDateAsStr(new Date(oldFile.lastModified())));
//        boolean status = oldFile.renameTo(backupFile);
//        if (!status) {
//          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                  "Unable to rename: " + oldFile + " to: " + backupFile);
//        }
//      }
//      boolean status = file.renameTo(oldFile);
//      if (status) {
//      } else {
//        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                "Unable to rename: " + file + " to: " + oldFile);
//      }
//    }
//  }
//
//  private String getDateAsStr(Date d) {
//    return new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.US).format(d);
//  }
//
//  /**
//   * If the index is stale by any chance, load index from a different dir in the data dir.
//   */
//  private boolean modifyIndexProps(String tmpIdxDirName) {
//    LOG.info("New index installed. Updating index properties...");
//    File idxprops = new File(solrCore.getDataDir() + "index.properties");
//    Properties p = new Properties();
//    if (idxprops.exists()) {
//      InputStream is = null;
//      try {
//        is = new FileInputStream(idxprops);
//        p.load(is);
//      } catch (Exception e) {
//        LOG.error("Unable to load index.properties");
//      } finally {
//        IOUtils.closeQuietly(is);
//      }
//    }
//    p.put("index", tmpIdxDirName);
//    FileOutputStream os = null;
//    try {
//      os = new FileOutputStream(idxprops);
//      p.store(os, "index properties");
//    } catch (Exception e) {
//      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//              "Unable to write index.properties", e);
//    } finally {
//      IOUtils.closeQuietly(os);
//    }
//      return true;
//  }
//
//  private final Map<String, FileInfo> confFileInfoCache = new HashMap<String, FileInfo>();
//
//  /**
//   * The local conf files are compared with the conf files in the master. If they are same (by checksum) do not copy.
//   *
//   * @param confFilesToDownload The list of files obtained from master
//   *
//   * @return a list of configuration files which have changed on the master and need to be downloaded.
//   */
//  private Collection<Map<String, Object>> getModifiedConfFiles(List<Map<String, Object>> confFilesToDownload) {
//    if (confFilesToDownload == null || confFilesToDownload.isEmpty())
//      return Collections.EMPTY_LIST;
//    //build a map with alias/name as the key
//    Map<String, Map<String, Object>> nameVsFile = new HashMap<String, Map<String, Object>>();
//    NamedList names = new NamedList();
//    for (Map<String, Object> map : confFilesToDownload) {
//      //if alias is present that is the name the file may have in the slave
//      String name = (String) (map.get(ALIAS) == null ? map.get(NAME) : map.get(ALIAS));
//      nameVsFile.put(name, map);
//      names.add(name, null);
//    }
//    //get the details of the local conf files with the same alias/name
//    List<Map<String, Object>> localFilesInfo = replicationHandler.getConfFileInfoFromCache(names, confFileInfoCache);
//    //compare their size/checksum to see if
//    for (Map<String, Object> fileInfo : localFilesInfo) {
//      String name = (String) fileInfo.get(NAME);
//      Map<String, Object> m = nameVsFile.get(name);
//      if (m == null) continue; // the file is not even present locally (so must be downloaded)
//      if (m.get(CHECKSUM).equals(fileInfo.get(CHECKSUM))) {
//        nameVsFile.remove(name); //checksums are same so the file need not be downloaded
//      }
//    }
//    return nameVsFile.isEmpty() ? Collections.EMPTY_LIST : nameVsFile.values();
//  }
//
//  /**
//   * Delete the directory tree recursively
//   */
//  static boolean delTree(File dir) {
//    if (dir == null || !dir.exists())
//      return false;
//    boolean isSuccess = true;
//    File contents[] = dir.listFiles();
//    if (contents != null) {
//      for (File file : contents) {
//        if (file.isDirectory()) {
//          boolean success = delTree(file);
//          if (!success) {
//            LOG.warn("Unable to delete directory : " + file);
//            isSuccess = false;
//          }
//        } else {
//          boolean success = file.delete();
//          if (!success) {
//            LOG.warn("Unable to delete file : " + file);
//            isSuccess = false;
//            return false;
//          }
//        }
//      }
//    }
//    return isSuccess && dir.delete();
//  }
//
//  /**
//   * Disable periodic polling
//   */
//  void disablePoll() {
//    pollDisabled.set(true);
//    LOG.info("inside disable poll, value of pollDisabled = " + pollDisabled);
//  }
//
//  /**
//   * Enable periodic polling
//   */
//  void enablePoll() {
//    pollDisabled.set(false);
//    LOG.info("inside enable poll, value of pollDisabled = " + pollDisabled);
//  }
//
//  /**
//   * Stops the ongoing pull
//   */
//  void abortPull() {
//    stop = true;
//  }
//
//  long getReplicationStartTime() {
//    return replicationStartTime;
//  }
//
//  List<Map<String, Object>> getConfFilesToDownload() {
//    //make a copy first because it can be null later
//    List<Map<String, Object>> tmp = confFilesToDownload;
//    //create a new instance. or else iterator may fail
//    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
//  }
//
//  List<Map<String, Object>> getConfFilesDownloaded() {
//    //make a copy first because it can be null later
//    List<Map<String, Object>> tmp = confFilesDownloaded;
//    // NOTE: it's safe to make a copy of a SynchronizedCollection(ArrayList)
//    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
//  }
//
//  List<Map<String, Object>> getFilesToDownload() {
//    //make a copy first because it can be null later
//    List<Map<String, Object>> tmp = filesToDownload;
//    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
//  }
//
//  List<Map<String, Object>> getFilesDownloaded() {
//    List<Map<String, Object>> tmp = filesDownloaded;
//    return tmp == null ? Collections.EMPTY_LIST : new ArrayList<Map<String, Object>>(tmp);
//  }
//
//  Map<String, Object> getCurrentFile() {
//    Map<String, Object> tmp = currentFile;
//    FileFetcher tmpFileFetcher = fileFetcher;
//    if (tmp == null)
//      return null;
//    tmp = new HashMap<String, Object>(tmp);
//    if (tmpFileFetcher != null)
//      tmp.put("bytesDownloaded", tmpFileFetcher.bytesDownloaded);
//    return tmp;
//  }
//
//  boolean isPollingDisabled() {
//    return pollDisabled.get();
//  }
//
//  Long getNextScheduledExecTime() {
//    Long nextTime = null;
//    if (executorStartTime > 0)
//      nextTime = executorStartTime + pollInterval;
//    return nextTime;
//  }
//
//  private static class ReplicationHandlerException extends InterruptedException {
//    public ReplicationHandlerException(String message) {
//      super(message);
//    }
//  }
//
//  /**
//   * The class acts as a client for ReplicationHandler.FileStream. It understands the protocol of wt=filestream
//   *
//   * @see org.apache.solr.handler.ReplicationHandler.FileStream
//   */
//  private class FileFetcher {
//    boolean includeChecksum = true;
//
//    private File copy2Dir;
//
//    String fileName;
//
//    String saveAs;
//
//    long size, lastmodified;
//
//    long bytesDownloaded = 0;
//
//    FileChannel fileChannel;
//    
//    private FileOutputStream fileOutputStream;
//
//    byte[] buf = new byte[1024 * 1024];
//
//    Checksum checksum;
//
//    File file;
//
//    int errorCount = 0;
//
//    private boolean isConf;
//
//    private PostMethod post;
//
//    private boolean aborted = false;
//
//    private Long indexVersion;
//
//    FileFetcher(File dir, Map<String, Object> fileDetails, String saveAs,
//                boolean isConf, long latestVersion) throws IOException {
//      this.copy2Dir = dir;
//      this.fileName = (String) fileDetails.get(NAME);
//      this.size = (Long) fileDetails.get(SIZE);
//      this.isConf = isConf;
//      this.saveAs = saveAs;
//      if(fileDetails.get(LAST_MODIFIED) != null){
//        lastmodified = (Long)fileDetails.get(LAST_MODIFIED);
//      }
//      indexVersion = latestVersion;
//
//      this.file = new File(copy2Dir, saveAs);
//      
//      this.fileOutputStream = new FileOutputStream(file);
//      this.fileChannel = this.fileOutputStream.getChannel();
//
//      if (includeChecksum)
//        checksum = new Adler32();
//    }
//
//    /**
//     * The main method which downloads file
//     */
//    void fetchFile() throws Exception {
//      try {
//        while (true) {
//          final FastInputStream is = getStream();
//          int result;
//          try {
//            //fetch packets one by one in a single request
//            result = fetchPackets(is);
//            if (result == 0 || result == NO_CONTENT) {
//              // if the file is downloaded properly set the
//              //  timestamp same as that in the server
//              if (file.exists() && lastmodified > 0)
//                file.setLastModified(lastmodified);
//              return;
//            }
//            //if there is an error continue. But continue from the point where it got broken
//          } finally {
//            IOUtils.closeQuietly(is);
//          }
//        }
//      } finally {
//        cleanup();
//        //if cleanup suceeds . The file is downloaded fully. do an fsync
//        fsyncService.submit(new Runnable(){
//          public void run() {
//            try {
//              FileUtils.sync(file);
//            } catch (IOException e) {
//              fsyncException = e;
//            }
//          }
//        });
//      }
//    }
//
//    private int fetchPackets(FastInputStream fis) throws Exception {
//      byte[] intbytes = new byte[4];
//      byte[] longbytes = new byte[8];
//      try {
//        while (true) {
//          if (stop) {
//            stop = false;
//            aborted = true;
//            throw new ReplicationHandlerException("User aborted replication");
//          }
//          long checkSumServer = -1;
//          fis.readFully(intbytes);
//          //read the size of the packet
//          int packetSize = readInt(intbytes);
//          if (packetSize <= 0) {
//            LOG.warn("No content recieved for file: " + currentFile);
//            return NO_CONTENT;
//          }
//          if (buf.length < packetSize)
//            buf = new byte[packetSize];
//          if (checksum != null) {
//            //read the checksum
//            fis.readFully(longbytes);
//            checkSumServer = readLong(longbytes);
//          }
//          //then read the packet of bytes
//          fis.readFully(buf, 0, packetSize);
//          //compare the checksum as sent from the master
//          if (includeChecksum) {
//            checksum.reset();
//            checksum.update(buf, 0, packetSize);
//            long checkSumClient = checksum.getValue();
//            if (checkSumClient != checkSumServer) {
//              LOG.error("Checksum not matched between client and server for: " + currentFile);
//              //if checksum is wrong it is a problem return for retry
//              return 1;
//            }
//          }
//          //if everything is fine, write down the packet to the file
//          fileChannel.write(ByteBuffer.wrap(buf, 0, packetSize));
//          bytesDownloaded += packetSize;
//          if (bytesDownloaded >= size)
//            return 0;
//          //errorcount is always set to zero after a successful packet
//          errorCount = 0;
//        }
//      } catch (ReplicationHandlerException e) {
//        throw e;
//      } catch (Exception e) {
//        LOG.warn("Error in fetching packets ", e);
//        //for any failure , increment the error count
//        errorCount++;
//        //if it fails for the same pacaket for   MAX_RETRIES fail and come out
//        if (errorCount > MAX_RETRIES) {
//          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                  "Fetch failed for file:" + fileName, e);
//        }
//        return ERR;
//      }
//    }
//
//    /**
//     * The webcontainer flushes the data only after it fills the buffer size. So, all data has to be read as readFully()
//     * other wise it fails. So read everything as bytes and then extract an integer out of it
//     */
//    private int readInt(byte[] b) {
//      return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16)
//              | ((b[2] & 0xff) << 8) | (b[3] & 0xff));
//
//    }
//
//    /**
//     * Same as above but to read longs from a byte array
//     */
//    private long readLong(byte[] b) {
//      return (((long) (b[0] & 0xff)) << 56) | (((long) (b[1] & 0xff)) << 48)
//              | (((long) (b[2] & 0xff)) << 40) | (((long) (b[3] & 0xff)) << 32)
//              | (((long) (b[4] & 0xff)) << 24) | ((b[5] & 0xff) << 16)
//              | ((b[6] & 0xff) << 8) | ((b[7] & 0xff));
//
//    }
//
//    /**
//     * cleanup everything
//     */
//    private void cleanup() {
//      try {
//        //close the FileOutputStream (which also closes the Channel)
//        fileOutputStream.close();
//      } catch (Exception e) {/* noop */
//          LOG.error("Error closing the file stream: "+ this.saveAs ,e);
//      }
//      try {
//        post.releaseConnection();
//      } catch (Exception e) {
//      }
//      if (bytesDownloaded != size) {
//        //if the download is not complete then
//        //delete the file being downloaded
//        try {
//          file.delete();
//        } catch (Exception e) {
//          LOG.error("Error deleting file in cleanup" + e.getMessage());
//        }
//        //if the failure is due to a user abort it is returned nomally else an exception is thrown
//        if (!aborted)
//          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                  "Unable to download " + fileName + " completely. Downloaded "
//                          + bytesDownloaded + "!=" + size);
//      }
//    }
//
//    /**
//     * Open a new stream using HttpClient
//     */
//    FastInputStream getStream() throws IOException {
//      post = new PostMethod(masterUrl);
//      //the method is command=filecontent
//      post.addParameter(COMMAND, CMD_GET_FILE);
//      //add the version to download. This is used to reserve the download
//      post.addParameter(CMD_INDEX_VERSION, indexVersion.toString());
//      if (isConf) {
//        //set cf instead of file for config file
//        post.addParameter(CONF_FILE_SHORT, fileName);
//      } else {
//        post.addParameter(FILE, fileName);
//      }
//      if (useInternal) {
//        post.addParameter(COMPRESSION, "true");
//      }
//      if (useExternal) {
//        post.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate"));
//      }
//      //use checksum
//      if (this.includeChecksum)
//        post.addParameter(CHECKSUM, "true");
//      //wt=filestream this is a custom protocol
//      post.addParameter("wt", FILE_STREAM);
//      // This happen if there is a failure there is a retry. the offset=<sizedownloaded> ensures that
//      // the server starts from the offset
//      if (bytesDownloaded > 0) {
//        post.addParameter(OFFSET, "" + bytesDownloaded);
//      }
//      myHttpClient.executeMethod(post);
//      InputStream is = post.getResponseBodyAsStream();
//      //wrap it using FastInputStream
//      if (useInternal) {
//        is = new InflaterInputStream(is);
//      } else if (useExternal) {
//        is = checkCompressed(post, is);
//      }
//      return new FastInputStream(is);
//    }
//  }
//
//  /*
//   * This is copied from CommonsHttpSolrServer
//   */
//  private InputStream checkCompressed(HttpMethod method, InputStream respBody) throws IOException {
//    Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
//    if (contentEncodingHeader != null) {
//      String contentEncoding = contentEncodingHeader.getValue();
//      if (contentEncoding.contains("gzip")) {
//        respBody = new GZIPInputStream(respBody);
//      } else if (contentEncoding.contains("deflate")) {
//        respBody = new InflaterInputStream(respBody);
//      }
//    } else {
//      Header contentTypeHeader = method.getResponseHeader("Content-Type");
//      if (contentTypeHeader != null) {
//        String contentType = contentTypeHeader.getValue();
//        if (contentType != null) {
//          if (contentType.startsWith("application/x-gzip-compressed")) {
//            respBody = new GZIPInputStream(respBody);
//          } else if (contentType.startsWith("application/x-deflate")) {
//            respBody = new InflaterInputStream(respBody);
//          }
//        }
//      }
//    }
//    return respBody;
//  }
//
//  static Integer readInterval(String interval) {
//    if (interval == null)
//      return null;
//    int result = 0;
//    if (interval != null) {
//      Matcher m = INTERVAL_PATTERN.matcher(interval.trim());
//      if (m.find()) {
//        String hr = m.group(1);
//        String min = m.group(2);
//        String sec = m.group(3);
//        result = 0;
//        try {
//          if (sec != null && sec.length() > 0)
//            result += Integer.parseInt(sec);
//          if (min != null && min.length() > 0)
//            result += (60 * Integer.parseInt(min));
//          if (hr != null && hr.length() > 0)
//            result += (60 * 60 * Integer.parseInt(hr));
//          result *= 1000;
//        } catch (NumberFormatException e) {
//          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                  INTERVAL_ERR_MSG);
//        }
//      } else {
//        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
//                INTERVAL_ERR_MSG);
//      }
//
//    }
//    return result;
//  }
//
//  public void destroy() {
//    if (executorService != null) executorService.shutdown();
//  }
//
//  String getMasterUrl() {
//    return masterUrl;
//  }
//
//  String getPollInterval() {
//    return pollIntervalStr;
//  }
//
//  private static final int MAX_RETRIES = 5;
//
//  private static final int NO_CONTENT = 1;
//
//  private static final int ERR = 2;
//
//  public static final String REPLICATION_PROPERTIES = "replication.properties";
//
//  public static final String POLL_INTERVAL = "pollInterval";
//
//  public static final String INTERVAL_ERR_MSG = "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'";
//
//  private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)");
//
//  private static final String HTTP_CONN_TIMEOUT = "httpConnTimeout";
//
//  private static final String HTTP_READ_TIMEOUT = "httpReadTimeout";
//
//  private static final String HTTP_BASIC_AUTH_USER = "httpBasicAuthUser";
//
//  private static final String HTTP_BASIC_AUTH_PASSWORD = "httpBasicAuthPassword";
//
//  static final String INDEX_REPLICATED_AT = "indexReplicatedAt";
//
//  static final String TIMES_INDEX_REPLICATED = "timesIndexReplicated";
//
//  static final String CONF_FILES_REPLICATED = "confFilesReplicated";
//
//  static final String CONF_FILES_REPLICATED_AT = "confFilesReplicatedAt";
//
//  static final String TIMES_CONFIG_REPLICATED = "timesConfigReplicated";
//
//  static final String LAST_CYCLE_BYTES_DOWNLOADED = "lastCycleBytesDownloaded";
//
//  static final String TIMES_FAILED = "timesFailed";
//
//  static final String REPLICATION_FAILED_AT = "replicationFailedAt";
//
//  static final String PREVIOUS_CYCLE_TIME_TAKEN = "previousCycleTimeInSeconds";
//
//  static final String INDEX_REPLICATED_AT_LIST = "indexReplicatedAtList";
//
//  static final String REPLICATION_FAILED_AT_LIST = "replicationFailedAtList";
//}
